package com.crawler.util;

import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 基于Jsoup的解析工具类
 *
 */
public class JsoupUtil {
	public static String[] forwardProfixs = new String[] {"http://www.baidu.com/link?", "http://t.cn"};
	public static String[] forwardContains = new String[] {".baidu.com/link?", "http://t.cn"};
	public static boolean isDebug = false;

	/**
	 * 将指定字符串解析为xml.
	 * @param xml
	 * @return
	 * @throws Exception
	 */
	public static Document parseXmlByJsoup(String xml) {
		Document document = Jsoup.parse(xml);
		return document;
	}

	/**
	 * 获取指定子元素的内容.
	 * @param element
	 * @param cssQuery
	 * @return
	 */
	public static String findElementText(Element element, String... cssQuerys) {
		if (cssQuerys != null && cssQuerys.length > 0) {
			for (String cssQuery : cssQuerys) {
				String elementText = getElementText(element, cssQuery);
				if (!elementText.isEmpty()) {
					return elementText;
				}
			}
		}
		return "";
	}

	/**
	 * 获取指定子元素的内容.
	 * @param element
	 * @param cssQuery
	 * @return
	 */
	public static String getElementText(Element element, String cssQuery) {
		if (element == null || cssQuery == null || cssQuery.isEmpty()) {
			return "";
		}
		Elements digestElement = element.select(cssQuery);
		if (digestElement != null) {
			return element2text(digestElement.first());
		}
		return "";
	}

	/**
	 * 获取Element中的文本信息.
	 * e.g. <span><em>sybn<em>123</span> ==> sybn123
	 * @param element
	 * @return
	 */
	public static String element2text(Element element) {
		if (element == null) {
			return "";
		}
		String asXML = element.html();
		if (asXML == null) {
			return "";
		}
		asXML = asXML.replace("\r", "");
		asXML = asXML.replace("\n", "");
		asXML = asXML.replace("&nbsp;", " ");
		asXML = asXML.replace("&ensp;", " ");
		asXML = asXML.replace("&emsp;", " ");
		asXML = asXML.replace("&#160;", " ");
		asXML = asXML.replaceAll("\\s{1,}", " ");
		return asXML.replaceAll("<.*?>", "");
	}

	/**
	 * 获取Element中的文本信息.并使用是定条件替换结�?.
	 * @param element
	 * @return
	 */
	public static String element2text(Element element, String regex,
			String replacement) {
		String text = element2text(element);
		return text.replaceAll(regex, replacement);
	}
	
	/**
	 * 使用指定条件,查找满足条件的第1个Element.
	 * 为null或为""则返回null;
	 * @param element
	 * @param cssQuery
	 * @return
	 */
	public static Element findElement(Element element, String cssQuery) {
		Elements urlElements = element.select(cssQuery);
		if (urlElements == null || urlElements.isEmpty()) {
			String asXML = element.html();
			if (!asXML.contains("qiyu-brand-title-icon")){
				//System.out.println("非法内容: " + asXML);
			}
			return null;
		}
		return urlElements.first();
	}
	/**
	 * 使用指定条件,查找满足条件的所有Element
	 * 
	 * @param element
	 * @param cssQuery
	 * @return Elements (implements List<Element>)
	 */
	public static Elements findElements(Element element, String cssQuery) {
		Elements urlElements = element.select(cssQuery);
		if (urlElements == null || urlElements.isEmpty()) {
			String asXML = element.html();
			if (!asXML.contains("qiyu-brand-title-icon")){
				System.out.println("非法内容: " + asXML);
			}
			return null;
		}
		return urlElements;
	}

	/**
	 * 打印目标内容
	 * @param key
	 * @param value
	 */
	public static void println(String key, String value) {
		if (!isDebug) {
			return;
		}
		String str = value;
		if (value.length() > 1024) {
			str = "length:" + value.length() + " >>> " + value.replaceAll("\\s{1,}", " ").substring(0, 1000) + "...";
		}
		System.out.println(key + ":" + str);
	}

	/**
	 * 获得真实的url地址.
	 * 有的引擎使用相对地址展示链接,我们�?要将其转换为绝对地址.
	 * 百度等引擎使用临时地�?展示链接,我们�?要将此临时地�?转换为真实地�?.
	 * @param baseUrl
	 * @param url
	 * @return
	 */
	public static String getLoction(String baseUrl, String url) {
		try {
			String urlLower = url.toLowerCase();
			if (urlLower.startsWith("//")) {
				url = "http:" + url;
				urlLower = url.toLowerCase();
			}
			if (!urlLower.startsWith("http")) {
				URI uri = new URI(baseUrl);
				URI resolve = uri.resolve(url);
				url = resolve.toString();
			}
			if (!isForwardUrl(url)) {
				return url;
			}

			URL serverUrl = new URL(url);
			HttpURLConnection conn = (HttpURLConnection) serverUrl
					.openConnection();
			conn.setRequestMethod("GET");
			// 必须设置false，否则会自动redirect到Location的地�?
			conn.setInstanceFollowRedirects(false);

			conn.addRequestProperty("Accept-Charset", "UTF-8;");
			conn.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8");
			conn.addRequestProperty("Referer", "http://zuidaima.com/");
			conn.connect();
			String location = conn.getHeaderField("Location");

			return location;

		} catch (Exception e) {
		}
		return "";
	}

	/**
	 * 查url是否为中转链.
	 * 比如百度的搜索结果或者微博的短连,都是中转链接.
	 * @param url
	 * @return
	 */
	private static boolean isForwardUrl(String url) {
		String lowerurl = url.toLowerCase();
		for (String forwardProfix : forwardProfixs) {
			if (lowerurl.startsWith(forwardProfix)) {
				return true;
			}
		}
		for (String forwardContain : forwardContains) {
			if (lowerurl.contains(forwardContain)) {
				return true;
			}
		}
		return false;
	}
	
	public static class HtmlFilter{
		public HtmlFilter(String regex, String replacement){
			this.regex = regex;
			this.replacement = replacement;
		}
		public HtmlFilter(String regex){
			this.regex = regex;
			this.replacement = "";
		}
		public String regex;
		public String replacement;
	}
	
	public static void main(String[] args) {
		String a = "<a href=\"/url?q=http://www.baike.com/wiki/%25E7%25BD%2597%25E7%25A3%258A&amp;sa=U&amp;ei=asxlVJzcEbaZsQT4x4AI&amp;ved=0CCgQFjAE&amp;usg=AFQjCNEJel08UnJbSOvCfgu8TZwRsWluDw\"><b>罗磊</b>_互动百科</a>";
		System.out.println(a);
		System.out.println(a.replaceAll("href=\"/url\\?q=([^\"]*)&amp;sa=[^\"]*\"", "href=\"$1\""));
		System.out.println(a.replaceAll("href=\"/url\\?q=([^\"]*)\"", "href=\"$1\""));
	}
}
